package gbench.sandbox.matlib;

import org.junit.jupiter.api.Test;

import java.util.*;
import java.util.function.*;
import java.util.stream.*;
import java.util.concurrent.atomic.AtomicInteger;

import gbench.common.fs.FileSystem;
import gbench.common.fs.XlsFile.DataMatrix;
import gbench.common.matlib.MatlibCanvas.NVec;
import gbench.common.matlib.MatlibCanvas.Range;
import gbench.common.matlib.MatlibCanvas.XVec;
import gbench.common.matlib.data.DataReader;
import gbench.common.matlib.data.DataReader.DFrame;
import gbench.common.matlib.data.DataReader.FVec;
import gbench.common.matlib.matrix.Matrix;
import gbench.common.tree.LittleTree.IRecord;

import static gbench.common.matlib.MatlibCanvas.println;
import static gbench.common.matlib.data.DataReader.DFrame.DF;
import static gbench.common.matlib.data.Pipe.DUMMY;
import static gbench.common.tree.LittleTree.IRecord.A;
import static gbench.common.tree.LittleTree.IRecord.REC;
import static gbench.common.tree.LittleTree.IRecord.subset;
import static gbench.common.tree.LittleTree.IRecord.test;
import static gbench.common.tree.LittleTree.IRecord.test2;
import static gbench.common.tree.LittleTree.IRecord.mapper;
import static gbench.common.tree.LittleTree.IRecord.predicate;
import static gbench.common.tree.LittleTree.IRecord.llclc;
import static gbench.common.tree.LittleTree.compose_f;
import static gbench.common.matlib.matrix.MatrixOps.Tuple2.TUP2;

import java.time.LocalDateTime;

/**
 *  数据框的使用演示
 * @author gbench
 *
 */
public class JunitDFrame {
    
    @Test
    final void foo() {
        final var users = DFrame.of( // 简单的用户数据表
            "id",NVec.of(4,i->i+1).ints(),
            "name", "张三,李四,王五,赵六".split(","), 
            "sex", "男女".toCharArray(), // 循环补填
            "weight", NVec.of(4,i->Math.random()*30+40).dblA(), // 循环补填
            "birth",Stream.of("1990-3-13,1990-5-28,1990-8-16,1990-10-5".split(","))
                .map(IRecord.coerce(LocalDateTime.class)).toArray(LocalDateTime[]::new),
            "married", NVec.of(4,i->i).map(e->Math.random()>0.5).toArray(Boolean[]::new),
            "country","中华人民共和国" // 循环补填
        );
        
        println(users);
        
        // 按照列读取
        println("列数据读取：");
        println("id",users.i4("id"));
        println("id lng",users.lng("id"));
        println("id lng",users.eval(rec->rec.lla("id",(Number num)->num.longValue())));
        println("id fx:手动类型变换", users.get("id", (Function<int[], ?>) aa -> Arrays.stream(aa).mapToObj(e -> (long) e).toArray(Long[]::new)));
        println("name str",users.str("name"));
        println("weight dbl",users.dbl("weight"));
        println("weight i4",users.i4("weight"));
        println("married bool",users.bool("married"));
        println("married i4",users.i4("married"));
        println("birth ldt",users.ldt("birth"));
        println("birth date",users.date("birth"));
        println("birth string",users.str("birth"));
        println("sex chr",users.chr("sex"));
        println("sex str",users.str("sex"));
        
        println("----------------------------------------------------------");
        println("sex 因子化",users.str("sex").collect(FVec.fveclc()));
        println("A-Z 的因子化",XVec.OF("ABCDEFGHIJKLMNOPQRSTUVWXYZ".split("")).collect(FVec.fveclc()));
        
        // 添加新的列
        println("----------------------------------------------------------");
        final var u2 = users.cbind( // 补充新的列
            "title","领队,向导,司机,厨师".split(","), // 职称
            "salary",NVec.of(4,i->Math.random()*1000) // 薪水)
        ); // u2
        println(u2);
        println("title",u2.objx("title"));
        
        // 添加新的行
        println("----------------------------------------------------------");
        final var u3 = u2.rbind1("5 陈七  女   67.1962 1990-10-05T00:00    false   中华人民共和国 会计  697.9109".split("\\s+"));
        final var u4 = u3.rbind1(IRecord.L("6 周八  男   67.1962 1990-10-05T00:00    false   中华人民共和国 电力工程师  697.9109".split("\\s+")));
        println(u4);
        println(u4.ldt("birth"));
        println(u4.fget("birth"));
        println(u4.fget("birth").pvec());
    }
    
    @Test
    final void loaddf() {
        final var datafile = FileSystem.path("data.txt", this.getClass());
        final var df = DataReader.loaddfs(datafile).get("table1");
        final var X = df.get("x");
        final var Y = df.get("y");
        final var sigma = df.get("sigma");
        System.out.println("X:"+X);
        System.out.println("Y:"+Y);
        System.out.println("sigma:"+sigma);
        
        System.out.println("Sxi:"+X.sum());
        System.out.println("Sxi2:"+X.fmapN(e->e*e).sum());
        System.out.println("Syi:"+Y.sum());
        System.out.println("Sxiyi:"+Y.mul(X).sum());
        System.out.println("deta:"+Y.mul(X).sum());
        
        System.out.println("读取数据框并转换");
        System.out.println(df.pipe().evaluate());
        System.out.println(df.pipe2().bind(DataMatrix::simpleOf).evaluate());
        
        System.out.println("遍历数据框");
        DataReader.loaddfs(datafile).forEach((name,d)->{
            System.out.println("\n#"+name);
            System.out.println(d);
        });
    }
    
    @Test
    public void mx() {
        final var rec = REC(
            "A", A(1,3,5), 
            "B", A(2,4,6,8));
        final var mx = DF(rec).pipe()
            .bind(e->e.toArray2(Integer.class))
            .bind(DataMatrix::simpleOf).evaluate();
        System.out.println("mx:\n"+mx);
        final var df = DF(rec);
        final var p_nvec = DUMMY(Integer[].class).bind(NVec::of);
        println("0",df.pipe(0,p_nvec));
        println("1",df.pipe(1,p_nvec).div2(2));
    }
    
    /**
     * 因子化读取
     * @throws InterruptedException
     */
    @Test
    public void factor() throws InterruptedException {
        final var datafile = FileSystem.path("data.txt", this.getClass());
        final var df = DataReader.loaddfs(datafile).get("table4");
        println("df\n",df);
        println("df.fget(\"sex\")",df.fget("sex"));
        println("df.fget(\"sex\").originals()",df.fget("sex").originals());
        println("df.fget(\"sex\").pvec()",df.fget("sex").pvec());
        println("df.fget(\"weight\")",df.fget("weight"));
        println("df.get(\"weight\")",df.get("weight"));
        println("df.toXVec()",df.toXVec()); // 计数因子
        final var ai = new AtomicInteger(0);
        println("df.toXVec(mapper)",df.toXVec(e->TUP2(ai.getAndIncrement(), e)));
        println("df.toXVecf()",df.toXVecf());
    }
    
    /**
     * 数据框的读取
     */
    @Test
    public void dataframe() {
        final var datafile = FileSystem.path("data.txt", this.getClass());
        final var iris = DataReader.loaddf(datafile, "iris");
        final var versicolor = iris.dataframe(subset(test("Species","versicolor"::equals)));
        System.out.println("versicolor:");
        System.out.println(versicolor);
        System.out.println("\niris.dataframe(subset(test2(\"Species\",\"versicolor\"::equals))):");
        System.out.println(iris.dataframe(subset(test2("Species","versicolor"::equals))));
        System.out.println("\ncolumnS:");
        iris.columnsS().forEach(System.out::println);
        System.out.println("\ncolS:");
        iris.colsS().forEach(System.out::println);
        System.out.println("\nfvecS:");
        iris.fvecS().forEach(System.out::println);
        System.out.println("\nrowS:");
        iris.rowsS().forEach(System.out::println);
    }
    
    /**
     * 数据矩阵的读取
     */
    @Test
    public void dfm() {
        final var file = FileSystem.fileOf("advertising.txt", this.getClass()); // 读取广告数据文件
        final var dfm = DataReader.loaddf(file,"advertising");// 读取数据: 0 广告数据, 1 模拟数据
        final var dfm_train = dfm  // 抽样训练数据
            // .shuffle() // 打乱顺序
            .head(100); // 提取100条数据用作训练
        println("dfm_train 训练数据集合\n", dfm_train);
        
        final var bundle = dfm_train.dfrm(rec -> rec.update((String k, List<String> col) -> { // 列类型转换
            return col.stream().map(IRecord.coerce(Double.class)).toArray(); // 转换为 Double 数组
        })).split(Range.of(4)).fmap(bp -> bp // 裂解的行项变换
            .fmap1(r -> r.arrayOf(Double.class,NVec::new).set(0, 1d)) // 提取一组元素,并在首位设置常量系数1
            .fmap2(r -> r.dbl(0)) // 只提取一个元素,生成一条值向量
        ).unwind(); // 纵向分解为向量束
        final var x_train = bundle._1().collect(Matrix.mxclc()); // 训练数据集合
        final var y_train = bundle._2().mutate(NVec::new); // 基准数据集合
        
        println("\nx_train 训练数据x矩阵:\n", x_train);
        println("\ny_train 训练数据实际值:\n", y_train.toString(e -> e + "\n"));
        
        final var drec = dfm.head(5).splitDREC(A(0, 1, 2, 3), A(1, 2), A(4)).alias("x_train,tvradio,y");
        println("x_train\n", drec.dfm("x_train"));
        println("tv\n", drec.dfm("tvradio"));
        println("y nvec", drec.nvec("y"));
        println("y xvec obj", drec.xvec("y"));
        println("y xvec int", drec.xvec("y", 0, Integer.class));
        println("非法值 xvec<int> of (sdfasf,100) ", drec.xvec("y", 100, Integer.class));
        println("非法值 xvec<int> of (y,0)", drec.xvec("sdfasf", 0, Integer.class));
    }
    
    /**
     * 行列绑定
     */
    @Test
    public void bind() {
        final var df = new DFrame("x",1,"y",1,"species","versicolor");
        final var df2 = new DFrame("x",2,"y",1,"species","virgina");
        System.out.println(df);
        System.out.println(df2);
        System.out.println(df.rbind(df2).rbind("sepal.length",100));
        System.out.println(new DFrame("id",NVec.of(1,2,3,4).intA()).cbind("name",IRecord.A("zhangsan","lisi")));
    }
    
    /**
     * DFrame的行过滤演示(sample 随机抽样)
     */
    @Test
    public void filter(){
        final var datafile = FileSystem.path("data.txt", this.getClass());
        final var iris = DataReader.loaddfs(datafile).get("iris");
        println(iris.rowsS().filter(e->Math.random()>0.9)
                .map(mapper("Species",(String t)->"species-"+t))
                .collect(IRecord.<String, DFrame>llclc(compose_f(e->new DFrame("Species",e), e->e.toArray(String[]::new)))));
        final var df = iris.rowsS().filter(e->Math.random()>0.7) // 选取30%(1-0.7)的行项目
                .collect(llclc(compose_f(DFrame::new, IRecord::ROWS2COLS))) // 合并成一个DFrame
                .rfilter(predicate("Species", "versicolor"::equals)); // 提取versicolor的分类
        println(df);
    }
    
    /**
     * DFrame的 行数据 归集(collect)演示
     */
    @Test
    public void collect(){
        final var datafile = FileSystem.path("data.txt", this.getClass());
        final var iris = DataReader.loaddfs(datafile).get("iris");
        final var aa = iris.eval(IRecord.ROWSCLC(IRecord.pvtclcS("Species",DFrame::new)));
        aa.columnsS()
        .map(col->col.car((Stream<IRecord>)null).collect(IRecord.llclc(IRecord::ROWS2COLS)).mutate(DFrame::new)) // 列归集成DFrame结构
        .forEach(df->{
            System.out.println(df); 
        });
    }
    
    /**
     * CSV 文件的读取
     */
    @Test
    public void csv() {
        final var data = XVec.OF( // CSV 文件的结构读取
            "-- iris1",  // 表 iris1
            "\"iris1-149\",6.2,3.4,5.4,2.3,\"virginica\"",
            "\"iris1-150,\\\"good,name\",5.9,3,5.1,1.8,\"virginica,good name\"",
            
            " -- iris2",  // 表 iris2 注意 -- 前有一个空格
            "\"iris2-149\",6.2,3.4,5.4,2.3,\"virginica\"",
            "\"iris2-150,\\\"good,name\",5.9,3,5.1,1.8,\"virginica,good name\""
            
        );// 注意这里的 第一个元素是 iris2-150,"good,name;
        
        println("按照表名进行检索");
        DataReader.getCSVLineS(data.stream()).map(XVec::OF).forEach(e -> { // 数据行分解
            println("line:",e,"size:",e.size());
        });
        
        println();
        println("按照表名进行检索");
        DataReader.getCSVLineS(data.stream(),"iris2").map(XVec::OF).forEach(e -> { // 数据行分解
            println("line:",e,"size:",e.size());
        });
    }

}
